/*
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
package net.javlov.example.chain;

import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.io.Serializable;
import java.io.StreamCorruptedException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.Map.Entry;

import net.javlov.Action;
import net.javlov.DecayingLearningRate;
import net.javlov.Option;
import net.javlov.Policy;
import net.javlov.SarsaAgent;
import net.javlov.Simulator;
import net.javlov.State;
import net.javlov.TDAgent;
import net.javlov.policy.RandomPolicy;
import net.javlov.vf.*;

public class MRPMain implements Runnable {

	Simulator sim;
	ValueMap<Integer> vt;
	
	public static void main(String[] args) {
		MRPMain m = new MRPMain();
		m.init();
		m.run();
	}
	
	public void init() {
		//params
		double gamma = 1, lambda = 0.9;
		
		//world
		Chain world = new Chain(10);
		
		//actions
		List<Action> optionPool = new ArrayList<Action>(2);
		Move left = new Move(world, Move.LEFT),
			right = new Move(world, Move.RIGHT);
		optionPool.add(left);
		optionPool.add(right);
		
		//value function
		vt = new ValueMap<Integer>(10); //for TD(0)
		TracedValueTable<Integer> tvt = new TracedValueTable<Integer>( vt,
									new BasicStateTrace(gamma, lambda, 1, true), 0.001);
		
		//policy
		Policy pi = new RandomPolicy(optionPool, new Random());
		
		//agent
		TDAgent agent = new TDAgent(tvt, gamma);
		agent.setPolicy(pi);
		agent.setLearnRate( new DecayingLearningRate(1,1,0.8) );
		
		//simulator / glue
		sim = new Simulator();
		sim.setAgent(agent);
		sim.setEnvironment(world);		
	}
	
	public void run() {
		ValueMap<Integer> convMap = (ValueMap)loadObject("convergedtable.ser");
		int episodes = 1000;
		sim.init();
		for ( int i = 0; i < episodes; i++ ) {
			sim.runEpisode();
			sim.reset();
			System.out.println(calcTableError(convMap, vt));
		}
		System.out.println(vt);
		//serializeObject(vt, "convergedtable.ser");
	}
	
	public <T> double calcTableError(ValueMap<T> convMap, ValueMap<T> newMap) {
		double err = 0;
		for ( Map.Entry<State<? extends T>, Value> svPair : convMap.stateValueSet() ) {
			err += Math.pow(svPair.getValue().get() - newMap.getValue(svPair.getKey()),2);
		}
		return err;
	}
	
	public static void serializeObject(Serializable obj, String fileName) {
        ObjectOutputStream oos = null;
        try {
                oos = new ObjectOutputStream( new FileOutputStream(fileName) );
                oos.writeObject(obj);
        } catch (IOException e) {
                System.out.println("Oops, an IOException occurred: " + e);
        } finally {
                try {
                        oos.close();
                } catch (IOException e) {
                        System.out.println("Oops, an IOException occurred: " + e);
                }
        }
	}

	public static Object loadObject(String filename) {
        Object obj = null;
        ObjectInputStream ois = null;
        try {
                ois = new ObjectInputStream( new FileInputStream(filename) );
                obj = ois.readObject();
        } catch (StreamCorruptedException sce) {
                System.err.println("Not a binary file: " + filename);
        } catch (Exception e) {
                System.err.println("WARNING: Problem trying to restore object from file: " + filename 
                                + ": " + e + ".");
        } finally {
                try {
                        if ( ois != null )
                                ois.close();
                } catch (IOException ioe) {
                        System.err.println("WARNING: problem closing stream: " + ioe);
                }
        }
        
        return obj;
	}
}
